In [1]:
import pandas as pd
import numpy as np
import datetime as dt
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
# Load the dataset
file_path = 'ecommerce_customer_data_large.csv'
df = pd.read_csv(file_path)
Data Preprocessing
In [2]:
df.isnull().sum()
Out[2]:
Customer ID 0 Purchase Date 0 Product Category 0 Product Price 0 Quantity 0 Total Purchase Amount 0 Payment Method 0 Customer Age 0 Returns 47382 Customer Name 0 Age 0 Gender 0 Churn 0 dtype: int64
In [3]:
df.shape
Out[3]:
(250000, 13)
In [33]:
df = df[df["Returns"] != 1.000]
df
Out[33]:
| Customer ID | Purchase Date | Product Category | Product Price | Quantity | Total Purchase Amount | Payment Method | Customer Age | Returns | Customer Name | Age | Gender | Churn | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 3 | 44605 | 2023-01-17 13:14:36 | Electronics | 396 | 3 | 937 | Cash | 31 | 0.0 | John Rivera | 31 | Female | 0 |
| 6 | 13738 | 2023-07-25 05:17:24 | Electronics | 205 | 1 | 2773 | Credit Card | 27 | NaN | Lauren Johnson | 27 | Female | 0 |
| 8 | 13738 | 2021-12-21 03:29:05 | Home | 12 | 2 | 2175 | Cash | 27 | NaN | Lauren Johnson | 27 | Female | 0 |
| 9 | 13738 | 2023-02-09 00:53:14 | Electronics | 40 | 4 | 4327 | Cash | 27 | 0.0 | Lauren Johnson | 27 | Female | 0 |
| 10 | 33969 | 2023-02-28 19:58:23 | Clothing | 410 | 3 | 5018 | Credit Card | 27 | NaN | Carol Allen | 27 | Male | 0 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 249991 | 16247 | 2020-06-24 17:29:27 | Electronics | 368 | 1 | 1720 | Credit Card | 22 | 0.0 | Mike Thompson | 22 | Male | 0 |
| 249994 | 39806 | 2021-08-01 04:43:12 | Electronics | 225 | 5 | 5293 | Credit Card | 60 | 0.0 | Dana Brown | 60 | Female | 0 |
| 249995 | 33807 | 2023-01-24 12:32:18 | Home | 436 | 1 | 3664 | Cash | 63 | 0.0 | Gabriel Williams | 63 | Male | 0 |
| 249997 | 28055 | 2022-11-10 17:11:57 | Electronics | 441 | 5 | 5296 | Cash | 63 | NaN | Lisa Johnson | 63 | Female | 0 |
| 249999 | 4148 | 2020-09-07 05:12:19 | Home | 307 | 5 | 3634 | Cash | 32 | 0.0 | Angela Norton | 32 | Male | 0 |
148524 rows × 13 columns
RFM Analysis is a technique used for customer segmentation. It allows grouping customers based on their purchase behavior, enabling the development of strategies for each group. Recency (How recently a customer has made a purchase) Frequency (How often a customer makes purchases) Monetary (How much money a customer spends)¶
In [4]:
df["Purchase Date"].max()
Out[4]:
'2023-09-13 18:42:49'
In [5]:
df['Total Price'] = df['Product Price'] * df['Quantity']
In [6]:
df = df.rename(columns={"Purchase Date": "Purchase_Date"})
df = df.rename(columns={"Customer ID": "Customer_ID"})
df = df.rename(columns={"Total Price": "Total_Price"})
df['Purchase_Date'] = pd.to_datetime(df['Purchase_Date'])
today_date = dt.datetime(2023, 9, 15)
In [7]:
rfm = df.groupby('Customer_ID').agg({'Purchase_Date': lambda Purchase_Date: (today_date - Purchase_Date.max()).days,
'Customer_ID': lambda Customer_ID: Customer_ID.value_counts(),
'Total_Price': lambda Total_Price: Total_Price.sum()})
rfm
Out[7]:
| Purchase_Date | Customer_ID | Total_Price | |
|---|---|---|---|
| Customer_ID | |||
| 1 | 289 | 3 | 5600 |
| 2 | 73 | 6 | 6459 |
| 3 | 223 | 4 | 3613 |
| 4 | 442 | 5 | 4339 |
| 5 | 425 | 5 | 2263 |
| ... | ... | ... | ... |
| 49996 | 360 | 7 | 6107 |
| 49997 | 389 | 2 | 1592 |
| 49998 | 14 | 10 | 8440 |
| 49999 | 357 | 6 | 4188 |
| 50000 | 123 | 7 | 3871 |
49661 rows × 3 columns
In [8]:
rfm.columns = ['Recency', 'Frequency', 'Monetary']
In [9]:
rfm["recency_score"] = pd.qcut(rfm['Recency'], 5, labels=[5, 4, 3, 2, 1])
# 0-100, 0-20, 20-40, 40-60, 60-80, 80-100
rfm["frequency_score"] = pd.qcut(rfm['Frequency'].rank(method="first"), 5, labels=[1, 2, 3, 4, 5])
rfm["monetary_score"] = pd.qcut(rfm['Monetary'], 5, labels=[1, 2, 3, 4, 5])
rfm["RFM_SCORE"] = (rfm['recency_score'].astype(str) +
rfm['frequency_score'].astype(str))
In [10]:
rfm.head()
Out[10]:
| Recency | Frequency | Monetary | recency_score | frequency_score | monetary_score | RFM_SCORE | |
|---|---|---|---|---|---|---|---|
| Customer_ID | |||||||
| 1 | 289 | 3 | 5600 | 2 | 1 | 5 | 21 |
| 2 | 73 | 6 | 6459 | 4 | 4 | 5 | 44 |
| 3 | 223 | 4 | 3613 | 3 | 2 | 3 | 32 |
| 4 | 442 | 5 | 4339 | 1 | 3 | 4 | 13 |
| 5 | 425 | 5 | 2263 | 2 | 3 | 2 | 23 |
In [11]:
seg_map = {
r'[1-2][1-2]': 'Hibernating',
r'[1-2][3-4]': 'At Risk',
r'[1-2]5': 'Cant Loose',
r'3[1-2]': 'About to sleep',
r'33': 'Need Attention',
r'[3-4][4-5]': 'Loyal Customers',
r'41': 'Promising',
r'51': 'New Customers',
r'[4-5][2-3]': 'Potential Loyalists',
r'5[4-5]': 'Champions'
}
In [12]:
rfm['segment'] = rfm['RFM_SCORE'].replace(seg_map, regex=True)
In [44]:
rfm
Out[44]:
| Recency | Frequency | Monetary | recency_score | frequency_score | monetary_score | RFM_SCORE | segment | |
|---|---|---|---|---|---|---|---|---|
| Customer_ID | ||||||||
| 1 | 289 | 3 | 5600 | 3 | 2 | 5 | 32 | About to sleep |
| 2 | 141 | 2 | 3121 | 4 | 1 | 4 | 41 | Promising |
| 3 | 223 | 4 | 3613 | 3 | 4 | 4 | 34 | Loyal Customers |
| 4 | 687 | 2 | 1335 | 1 | 1 | 2 | 11 | Hibernating |
| 5 | 425 | 2 | 1033 | 2 | 1 | 2 | 21 | Hibernating |
| ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 49996 | 360 | 5 | 3974 | 3 | 5 | 5 | 35 | Loyal Customers |
| 49997 | 389 | 2 | 1592 | 2 | 2 | 2 | 22 | Hibernating |
| 49998 | 117 | 4 | 3957 | 4 | 5 | 5 | 45 | Loyal Customers |
| 49999 | 415 | 3 | 2810 | 2 | 4 | 4 | 24 | At Risk |
| 50000 | 123 | 4 | 2424 | 4 | 5 | 3 | 45 | Loyal Customers |
47490 rows × 8 columns
In [13]:
import pandas as pd
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
import seaborn as sns
# Assuming 'rfm' is the DataFrame with Recency, Frequency, and Monetary features
# Step 1: Preprocessing - Scaling the data
features = ['Recency', 'Frequency', 'Monetary']
X = rfm[features]
# Standardize the data
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
# Step 2: Determine the optimal number of clusters using the Elbow method
inertia = []
K = range(1, 11)
for k in K:
kmeans = KMeans(n_clusters=k, random_state=42)
kmeans.fit(X_scaled)
inertia.append(kmeans.inertia_)
# Plot the Elbow method
plt.figure(figsize=(8, 5))
plt.plot(K, inertia, 'bo-', markerfacecolor='r')
plt.title('Elbow Method For Optimal K')
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.show()
# Step 3: Fit K-Means with the optimal number of clusters (let's assume K=4 based on the elbow plot)
kmeans = KMeans(n_clusters=4, random_state=42)
rfm['Cluster'] = kmeans.fit_predict(X_scaled)
# Step 4: Analyze the clusters
rfm['Cluster'] = rfm['Cluster'].astype(str) # Convert cluster labels to string for easier visualization
plt.figure(figsize=(10, 7))
sns.scatterplot(data=rfm, x='Recency', y='Monetary', hue='Cluster', palette='Set1')
plt.title('Customer Segments based on Recency and Monetary')
plt.show()
# Step 5: Interpretation - View cluster means to understand segments
cluster_summary = rfm.groupby('Cluster').agg({
'Recency': 'mean',
'Frequency': 'mean',
'Monetary': 'mean'
}).reset_index()
print(cluster_summary)
Cluster Recency Frequency Monetary 0 0 195.805647 3.470939 2239.454886 1 1 141.140015 8.391029 7298.577293 2 2 720.527080 2.956077 2182.198238 3 3 179.217648 5.809469 4469.535728
In [14]:
import pandas as pd
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import silhouette_score, davies_bouldin_score
import matplotlib.pyplot as plt
import seaborn as sns
rfm['Cluster'] = rfm['Cluster'].astype(int)
silhouette_avg = silhouette_score(X_scaled, rfm['Cluster'])
davies_bouldin_avg = davies_bouldin_score(X_scaled, rfm['Cluster'])
print(f'Silhouette Score: {silhouette_avg:.3f}')
print(f'Davies-Bouldin Index: {davies_bouldin_avg:.3f}')
# Step 3: Visualize clusters with Pair Plot for detailed analysis
sns.pairplot(rfm, hue='Cluster', vars=features, palette='Set1')
plt.suptitle('Pair Plot of Clusters', y=1.02)
plt.show()
Silhouette Score: 0.311 Davies-Bouldin Index: 1.016
Business Model
In [15]:
rfm.head()
Out[15]:
| Recency | Frequency | Monetary | recency_score | frequency_score | monetary_score | RFM_SCORE | segment | Cluster | |
|---|---|---|---|---|---|---|---|---|---|
| Customer_ID | |||||||||
| 1 | 289 | 3 | 5600 | 2 | 1 | 5 | 21 | Hibernating | 3 |
| 2 | 73 | 6 | 6459 | 4 | 4 | 5 | 44 | Loyal Customers | 3 |
| 3 | 223 | 4 | 3613 | 3 | 2 | 3 | 32 | About to sleep | 0 |
| 4 | 442 | 5 | 4339 | 1 | 3 | 4 | 13 | At Risk | 3 |
| 5 | 425 | 5 | 2263 | 2 | 3 | 2 | 23 | At Risk | 0 |
In [16]:
# Assuming rfm DataFrame is already defined in your previous notebook
# Check the data type of the 'Cluster' column
print("Initial Cluster Data Type:", rfm['Cluster'].dtype)
# If it's not an integer type, convert it
if rfm['Cluster'].dtype == 'object':
rfm['Cluster'] = rfm['Cluster'].astype(int)
# Confirm the data types of the DataFrame
print("Data Types After Conversion:\n", rfm.dtypes)
# Function to assign business strategy and customer characteristics based on cluster
def assign_business_strategy(row):
if row['Cluster'] == 0: # Loyal Champions (Cluster 0)
return (
"Highly engaged, made recent purchases, high spending.",
[
"Offer loyalty rewards (e.g., VIP programs, early access to new products).",
"Introduce referral programs to encourage bringing in new customers.",
"Provide exclusive discounts or personalized product recommendations."
]
)
elif row['Cluster'] == 1: # At Risk (Cluster 1)
return (
"Haven't made a purchase recently but were frequent buyers.",
[
"Launch a re-engagement campaign with personalized offers and incentives.",
"Use email marketing to highlight new arrivals or special deals to win them back.",
"Offer time-sensitive discounts to create urgency and bring them back."
]
)
elif row['Cluster'] == 2: # New & Promising (Cluster 2)
return (
"New customers, recent engagement, low purchase frequency.",
[
"Focus on nurturing these customers through welcome offers or first-time buyer discounts.",
"Provide a personalized shopping experience to encourage repeat purchases.",
"Send follow-up emails or push notifications to keep them engaged."
]
)
elif row['Cluster'] == 3: # Hibernating Customers (Cluster 3)
return (
"Low purchase frequency, haven't bought anything in a long time.",
[
"Allocate minimal marketing resources to this segment.",
"Send occasional newsletters or low-investment offers (e.g., free shipping) to reawaken interest."
]
)
# Default return in case no match found
return (
"Unknown",
["No specific strategy assigned."]
)
# Apply the strategy assignment function to each row of the rfm DataFrame
rfm[['Customer_Characteristics', 'Business_Strategy_List']] = rfm.apply(
lambda row: pd.Series(assign_business_strategy(row)), axis=1
)
# Display relevant columns to verify the results
print(rfm[['Cluster', 'Customer_Characteristics', 'Business_Strategy_List']])
# Debugging output to ensure clusters are processed correctly
for i, row in rfm.iterrows():
print(f"Row {i}: Cluster = {row['Cluster']}, Characteristics = {row['Customer_Characteristics']}, Strategy = {row['Business_Strategy_List']}")